{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.04578213,  0.02354491,  0.03440884, -0.01564381], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAApGUlEQVR4nO3de3TU9Z3/8ddMLsMlzMQAySSSIAoFIgS7gGHW1qVLSrhopcbfUcsCdjlwZBNPJZZiWipi9xgX96yXLsIf64p7jpSWHtGVChaDhKoBNSXloqbCjzZYMglCM8NFcpn5/P5wmd+OQsiEMPMJeT7O+Z6T+X4+8/2+v5+TMC8+38s4jDFGAAAAFnEmugAAAIAvI6AAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOskNKCsWbNG1113nfr166fCwkK99957iSwHAABYImEB5Ze//KXKy8u1cuVK/f73v9eECRNUXFys5ubmRJUEAAAs4UjUlwUWFhZq8uTJ+vd//3dJUjgcVm5urh544AE9/PDDiSgJAABYIjkRO21ra1Ntba0qKioi65xOp4qKilRTU/OV/q2trWptbY28DofDOnnypAYPHiyHwxGXmgEAwOUxxujUqVPKycmR09n5SZyEBJTPPvtMoVBIWVlZUeuzsrL08ccff6V/ZWWlVq1aFa/yAADAFXT06FENGzas0z4JCSixqqioUHl5eeR1IBBQXl6ejh49KrfbncDKAABAVwWDQeXm5mrQoEGX7JuQgDJkyBAlJSWpqakpan1TU5O8Xu9X+rtcLrlcrq+sd7vdBBQAAHqZrlyekZC7eFJTUzVx4kRVVVVF1oXDYVVVVcnn8yWiJAAAYJGEneIpLy/XggULNGnSJN188816+umndebMGX3/+99PVEkAAMASCQsod999t44fP65HHnlEfr9fN910k7Zt2/aVC2cBAEDfk7DnoFyOYDAoj8ejQCDANSgAAPQSsXx+8108AADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADW6fGA8uijj8rhcEQtY8aMibSfO3dOpaWlGjx4sNLS0lRSUqKmpqaeLgMAAPRiV2QG5cYbb1RjY2NkefvttyNtS5cu1WuvvaZNmzapurpax44d05133nklygAAAL1U8hXZaHKyvF7vV9YHAgE9//zz2rBhg/7+7/9ekvTCCy9o7Nix2r17t6ZMmXIlygEAAL3MFZlB+eSTT5STk6Prr79ec+fOVUNDgySptrZW7e3tKioqivQdM2aM8vLyVFNTc9Httba2KhgMRi0AAODq1eMBpbCwUOvXr9e2bdu0du1aHTlyRN/85jd16tQp+f1+paamKj09Peo9WVlZ8vv9F91mZWWlPB5PZMnNze3psgEAgEV6/BTPzJkzIz8XFBSosLBQw4cP169+9Sv179+/W9usqKhQeXl55HUwGCSkAABwFbvitxmnp6fra1/7mg4dOiSv16u2tja1tLRE9WlqarrgNSvnuVwuud3uqAUAAFy9rnhAOX36tA4fPqzs7GxNnDhRKSkpqqqqirTX19eroaFBPp/vSpcCAAB6iR4/xfPDH/5Qt99+u4YPH65jx45p5cqVSkpK0r333iuPx6OFCxeqvLxcGRkZcrvdeuCBB+Tz+biDBwAARPR4QPn0009177336sSJExo6dKi+8Y1vaPfu3Ro6dKgk6amnnpLT6VRJSYlaW1tVXFys5557rqfLAAAAvZjDGGMSXUSsgsGgPB6PAoEA16MAANBLxPL5zXfxAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsE3NA2bVrl26//Xbl5OTI4XDolVdeiWo3xuiRRx5Rdna2+vfvr6KiIn3yySdRfU6ePKm5c+fK7XYrPT1dCxcu1OnTpy/rQAAAwNUj5oBy5swZTZgwQWvWrLlg++rVq/Xss89q3bp12rNnjwYOHKji4mKdO3cu0mfu3Lk6ePCgtm/fri1btmjXrl1avHhx948CAABcVRzGGNPtNzsc2rx5s+bMmSPpi9mTnJwcPfTQQ/rhD38oSQoEAsrKytL69et1zz336KOPPlJ+fr7ef/99TZo0SZK0bds2zZo1S59++qlycnIuud9gMCiPx6NAICC3293d8gEAQBzF8vndo9egHDlyRH6/X0VFRZF1Ho9HhYWFqqmpkSTV1NQoPT09Ek4kqaioSE6nU3v27LngdltbWxUMBqMWAABw9erRgOL3+yVJWVlZUeuzsrIibX6/X5mZmVHtycnJysjIiPT5ssrKSnk8nsiSm5vbk2UDAADL9Iq7eCoqKhQIBCLL0aNHE10SAAC4gno0oHi9XklSU1NT1PqmpqZIm9frVXNzc1R7R0eHTp48GenzZS6XS263O2oBAABXrx4NKCNGjJDX61VVVVVkXTAY1J49e+Tz+SRJPp9PLS0tqq2tjfTZsWOHwuGwCgsLe7IcAADQSyXH+obTp0/r0KFDkddHjhxRXV2dMjIylJeXpwcffFD//M//rFGjRmnEiBH66U9/qpycnMidPmPHjtWMGTO0aNEirVu3Tu3t7SorK9M999zTpTt4AADA1S/mgPLBBx/oW9/6VuR1eXm5JGnBggVav369fvSjH+nMmTNavHixWlpa9I1vfEPbtm1Tv379Iu956aWXVFZWpmnTpsnpdKqkpETPPvtsDxwOAAC4GlzWc1ASheegAADQ+yTsOSgAAAA9gYACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6MQeUXbt26fbbb1dOTo4cDodeeeWVqPb77rtPDocjapkxY0ZUn5MnT2ru3Llyu91KT0/XwoULdfr06cs6EAAAcPWIOaCcOXNGEyZM0Jo1ay7aZ8aMGWpsbIwsv/jFL6La586dq4MHD2r79u3asmWLdu3apcWLF8dePQAAuColx/qGmTNnaubMmZ32cblc8nq9F2z76KOPtG3bNr3//vuaNGmSJOnnP/+5Zs2apX/9139VTk5OrCUBAICrzBW5BmXnzp3KzMzU6NGjtWTJEp04cSLSVlNTo/T09Eg4kaSioiI5nU7t2bPngttrbW1VMBiMWgAAwNWrxwPKjBkz9F//9V+qqqrSv/zLv6i6ulozZ85UKBSSJPn9fmVmZka9Jzk5WRkZGfL7/RfcZmVlpTweT2TJzc3t6bIBAIBFYj7Fcyn33HNP5Ofx48eroKBAN9xwg3bu3Klp06Z1a5sVFRUqLy+PvA4Gg4QUAACuYlf8NuPrr79eQ4YM0aFDhyRJXq9Xzc3NUX06Ojp08uTJi1634nK55Ha7oxYAAHD1uuIB5dNPP9WJEyeUnZ0tSfL5fGppaVFtbW2kz44dOxQOh1VYWHilywEAAL1AzKd4Tp8+HZkNkaQjR46orq5OGRkZysjI0KpVq1RSUiKv16vDhw/rRz/6kUaOHKni4mJJ0tixYzVjxgwtWrRI69atU3t7u8rKynTPPfdwBw8AAJAkOYwxJpY37Ny5U9/61re+sn7BggVau3at5syZo71796qlpUU5OTmaPn26fvaznykrKyvS9+TJkyorK9Nrr70mp9OpkpISPfvss0pLS+tSDcFgUB6PR4FAgNM9AAD0ErF8fsccUGxAQAEAoPeJ5fOb7+IBAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOvE/GWBANBT/vy7DWo9faLTPtdOvkMDh+TFqSIAtiCgAEiIcDikYGO9zv21sdN+WeOmxakiADbhFA+AhDChDqn3fVcpgDghoABICAIKgM4QUAAkRDjcISMCCoALI6AASAgTCjGDAuCiCCgAEsKEOcUD4OIIKAASglM8ADpDQAGQEFwkC6AzBBQACUFAAdAZAgqAhAiHQ5ziAXBRBBQACcEMCoDOEFAAJIQJd8gQUABcBAEFQEKca/Er3NHWaZ/UgdcoydU/ThUBsAkBBUBCnP2sQeH2c5326XdNjlL6DYpTRQBsQkABYC2HM0ly8M8U0Bfxlw/AWo6kJDmcjkSXASABCCgArOVkBgXos/jLB2AthzNZDgIK0Cfxlw/AWg5nkhwOTvEAfVFMAaWyslKTJ0/WoEGDlJmZqTlz5qi+vj6qz7lz51RaWqrBgwcrLS1NJSUlampqiurT0NCg2bNna8CAAcrMzNSyZcvU0dFx+UcD4KrCRbJA3xXTX351dbVKS0u1e/dubd++Xe3t7Zo+fbrOnDkT6bN06VK99tpr2rRpk6qrq3Xs2DHdeeedkfZQKKTZs2erra1N7777rl588UWtX79ejzzySM8dFYCrAjMoQN/lMJfxKMfjx48rMzNT1dXVuvXWWxUIBDR06FBt2LBBd911lyTp448/1tixY1VTU6MpU6Zo69atuu2223Ts2DFlZWVJktatW6fly5fr+PHjSk1NveR+g8GgPB6PAoGA3G53d8sHkED/d8fzOvHJnk77ZI0v0rWT71BSiitOVQG4kmL5/L6sudNAICBJysjIkCTV1taqvb1dRUVFkT5jxoxRXl6eampqJEk1NTUaP358JJxIUnFxsYLBoA4ePHjB/bS2tioYDEYtAK5+X8ygcIoH6Iu6/ZcfDof14IMP6pZbbtG4ceMkSX6/X6mpqUpPT4/qm5WVJb/fH+nzv8PJ+fbzbRdSWVkpj8cTWXJzc7tbNoBexJGUJHGKB+iTuh1QSktLdeDAAW3cuLEn67mgiooKBQKByHL06NErvk8AiccMCtB3JXfnTWVlZdqyZYt27dqlYcOGRdZ7vV61tbWppaUlahalqalJXq830ue9996L2t75u3zO9/kyl8sll4tz0MDVoquXvjmcycygAH1UTP81McaorKxMmzdv1o4dOzRixIio9okTJyolJUVVVVWRdfX19WpoaJDP55Mk+Xw+7d+/X83NzZE+27dvl9vtVn5+/uUcC4BewpiwTDjcpb7cxQP0TTHNoJSWlmrDhg169dVXNWjQoMg1Ix6PR/3795fH49HChQtVXl6ujIwMud1uPfDAA/L5fJoyZYokafr06crPz9e8efO0evVq+f1+rVixQqWlpcySAH1FOCxjuhZQAPRNMQWUtWvXSpKmTp0atf6FF17QfffdJ0l66qmn5HQ6VVJSotbWVhUXF+u5556L9E1KStKWLVu0ZMkS+Xw+DRw4UAsWLNBjjz12eUcCoNcwJkRAAdCpy3oOSqLwHBSgd+toPasjb72glj//odN+1978XeV8fWacqgJwpcXtOSgA0B0mzAwKgM4RUADEnQmHunyRLIC+iYACIO6MCUsmlOgyAFiMgAIg7phBAXApBBQAccc1KAAuhYACIP7CXX9QG4C+iYACIO6+uAaFgALg4ggoAOKOUzwALoWAAiDuzn7WoM//2thpH9egIRo4JC9OFQGwDQEFQNyFOtpkQu2d9nEmu5SU2j9OFQGwDQEFgJ2cTjmcSYmuAkCCEFAAWMnhcBBQgD6MgALASg4HMyhAX0ZAAWAnp1MOJ/9EAX0Vf/0ArORwOOVwMIMC9FUEFABW4hQP0LcRUADYibt4gD6NgALASg6HU+IaFKDP4q8fgJUcDqeczKAAfRYBBYCdOMUD9GkEFABW+uIUDwEF6KsIKADiyhgjGXPJfg6H44uQAqBP4q8fQJwZmXCoC/0ccjgcV7waAHYioACIL9PVgAKgLyOgAIgrY8Iy4Y5ElwHAcgQUAHFljFGYGRQAl0BAARBfJswpHgCXREABEF/GyIQ4xQOgcwQUAHFluEgWQBcQUADEF6d4AHQBAQVAXBlO8QDogpgCSmVlpSZPnqxBgwYpMzNTc+bMUX19fVSfqVOn/s8TIP//cv/990f1aWho0OzZszVgwABlZmZq2bJl6ujgHyygT+AUD4AuSI6lc3V1tUpLSzV58mR1dHToxz/+saZPn64PP/xQAwcOjPRbtGiRHnvsscjrAQMGRH4OhUKaPXu2vF6v3n33XTU2Nmr+/PlKSUnR448/3gOHBMBmhlM8ALogpoCybdu2qNfr169XZmamamtrdeutt0bWDxgwQF6v94Lb+O1vf6sPP/xQb775prKysnTTTTfpZz/7mZYvX65HH31Uqamp3TgMAL2GMQrzoDYAl3BZ16AEAgFJUkZGRtT6l156SUOGDNG4ceNUUVGhs2fPRtpqamo0fvx4ZWVlRdYVFxcrGAzq4MGDF9xPa2urgsFg1AKgdwp3tKn9bOd/ww5nklIGuONUEQAbxTSD8r+Fw2E9+OCDuuWWWzRu3LjI+u9973saPny4cnJytG/fPi1fvlz19fV6+eWXJUl+vz8qnEiKvPb7/RfcV2VlpVatWtXdUgFYpP3zoM4e/1OnfZwp/eQelh+fggBYqdsBpbS0VAcOHNDbb78dtX7x4sWRn8ePH6/s7GxNmzZNhw8f1g033NCtfVVUVKi8vDzyOhgMKjc3t3uFA7Cew+GQw5mS6DIAJFC3TvGUlZVpy5YteuuttzRs2LBO+xYWFkqSDh06JEnyer1qamqK6nP+9cWuW3G5XHK73VELgKuZQ86kpEQXASCBYgooxhiVlZVp8+bN2rFjh0aMGHHJ99TV1UmSsrOzJUk+n0/79+9Xc3NzpM/27dvldruVn8+ULgBJDoccSd2e4AVwFYjpX4DS0lJt2LBBr776qgYNGhS5ZsTj8ah///46fPiwNmzYoFmzZmnw4MHat2+fli5dqltvvVUFBQWSpOnTpys/P1/z5s3T6tWr5ff7tWLFCpWWlsrlcvX8EQLodRySHEmc4gH6sphmUNauXatAIKCpU6cqOzs7svzyl7+UJKWmpurNN9/U9OnTNWbMGD300EMqKSnRa6+9FtlGUlKStmzZoqSkJPl8Pv3DP/yD5s+fH/XcFAB9nMMhp5NTPEBfFtMMijGm0/bc3FxVV1dfcjvDhw/X66+/HsuuAfQlnOIB+jy+iweAhRxyElCAPo2AAsA6DoeDa1CAPo6AAsBCDjmczKAAfRkBBYB9HOIUD9DHEVAAWMchhxzcxQP0aQQUAHFjjFHn9wL+DwcBBejrCCgA4sqE2rvUz+FwXOFKANiMgAIgrsKhjkSXAKAXIKAAiCPT5RkUAH0bAQVA/BjJMIMCoAsIKADiyCjcwQwKgEsjoACIK65BAdAVBBQAcWMkmTAzKAAujYACIH6MYQYFQJcQUADEFXfxAOgKAgqAODIKE1AAdAEBBUD8GCncwSkeAJdGQAEQV5ziAdAVBBQAcRMOdyjw6cFL9vMML4hDNQBslpzoAgD0HqFQSMZ06fuIL6ijvV1tp05copdDLrdXHZdxKsjpdMrp5P9fQG/GXzCALps/f7769+/f7SUnJ/uS+wiHw1q4aPFl7ecnP/lJHEYDwJXEDAqALguFQpc1s9HRcen/ExkZfd7adln7CYVC3X4vADsQUAAkRKBjsP7anq22sEsu5+canPIXpSUHJElt7QQMoK8joACIu6bW4frj2cn6PDRIISUrydGugc6g8tPekTupUa1t3IoM9HVcgwIgjhz6a3uW9p2eqtOhDIWUIsmhkElVMDREtcFinQ151NrBDArQ1xFQAMRNezhVuwPfUYdxXbjd9NPv/vp/1NoejnNlAGxDQAEQZ45OW40cauUaFKDPI6AAsA4BBQABBYBljNraCChAX0dAARA3Kc42TXK/LqcuHECc6pDP84raOtriXBkA28QUUNauXauCggK53W653W75fD5t3bo10n7u3DmVlpZq8ODBSktLU0lJiZqamqK20dDQoNmzZ2vAgAHKzMzUsmXLLuuBTAB6E6MhKZ9q/KCd6u88Jac6JBk51aEBzoAmubfKnfwZz0EBENtzUIYNG6YnnnhCo0aNkjFGL774ou644w7t3btXN954o5YuXarf/OY32rRpkzwej8rKynTnnXfqnXfekfTF0x1nz54tr9erd999V42NjZo/f75SUlL0+OOPX5EDBGCP9o6wXn3nY0kf66/tH+h4W67aTH/1c55RZuqf9dfkEwqHjdpD3MUD9HUOcznf/CUpIyNDTz75pO666y4NHTpUGzZs0F133SVJ+vjjjzV27FjV1NRoypQp2rp1q2677TYdO3ZMWVlZkqR169Zp+fLlOn78uFJTU7u0z2AwKI/Ho/vuu6/L7wFw+aqqqnT48OFEl3FJEyZMUGFhYaLLAPAlbW1tWr9+vQKBgNxud6d9u/0k2VAopE2bNunMmTPy+Xyqra1Ve3u7ioqKIn3GjBmjvLy8SECpqanR+PHjI+FEkoqLi7VkyRIdPHhQX//61y+4r9bWVrW2tkZeB4NBSdK8efOUlpbW3UMAEKMjR470ioBSUFCghQsXJroMAF9y+vRprV+/vkt9Yw4o+/fvl8/n07lz55SWlqbNmzcrPz9fdXV1Sk1NVXp6elT/rKws+f1+SZLf748KJ+fbz7ddTGVlpVatWvWV9ZMmTbpkAgPQczIyMhJdQpd4vV7dfPPNiS4DwJecn2Doipjv4hk9erTq6uq0Z88eLVmyRAsWLNCHH34Y62ZiUlFRoUAgEFmOHj16RfcHAAASK+YZlNTUVI0cOVKSNHHiRL3//vt65plndPfdd6utrU0tLS1RsyhNTU3yer2SvvhfzXvvvRe1vfN3+ZzvcyEul0su14UfjQ0AAK4+l/0clHA4rNbWVk2cOFEpKSmqqqqKtNXX16uhoUE+n0+S5PP5tH//fjU3N0f6bN++XW63W/n5+ZdbCgAAuErENINSUVGhmTNnKi8vT6dOndKGDRu0c+dOvfHGG/J4PFq4cKHKy8uVkZEht9utBx54QD6fT1OmTJEkTZ8+Xfn5+Zo3b55Wr14tv9+vFStWqLS0lBkSAAAQEVNAaW5u1vz589XY2CiPx6OCggK98cYb+va3vy1Jeuqpp+R0OlVSUqLW1lYVFxfrueeei7w/KSlJW7Zs0ZIlS+Tz+TRw4EAtWLBAjz32WM8eFQAA6NViCijPP/98p+39+vXTmjVrtGbNmov2GT58uF5//fVYdgsAAPoYvosHAABYh4ACAACsQ0ABAADWIaAAAADrdPu7eAD0PZMnT476XixbjRs3LtElALhMl/1txolw/tuMu/JtiAAAwA6xfH5zigcAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALBOTAFl7dq1KigokNvtltvtls/n09atWyPtU6dOlcPhiFruv//+qG00NDRo9uzZGjBggDIzM7Vs2TJ1dHT0zNEAAICrQnIsnYcNG6YnnnhCo0aNkjFGL774ou644w7t3btXN954oyRp0aJFeuyxxyLvGTBgQOTnUCik2bNny+v16t1331VjY6Pmz5+vlJQUPf744z10SAAAoLdzGGPM5WwgIyNDTz75pBYuXKipU6fqpptu0tNPP33Bvlu3btVtt92mY8eOKSsrS5K0bt06LV++XMePH1dqamqX9hkMBuXxeBQIBOR2uy+nfAAAECexfH53+xqUUCikjRs36syZM/L5fJH1L730koYMGaJx48apoqJCZ8+ejbTV1NRo/PjxkXAiScXFxQoGgzp48OBF99Xa2qpgMBi1AACAq1dMp3gkaf/+/fL5fDp37pzS0tK0efNm5efnS5K+973vafjw4crJydG+ffu0fPly1dfX6+WXX5Yk+f3+qHAiKfLa7/dfdJ+VlZVatWpVrKUCAIBeKuaAMnr0aNXV1SkQCOjXv/61FixYoOrqauXn52vx4sWRfuPHj1d2dramTZumw4cP64Ybbuh2kRUVFSovL4+8DgaDys3N7fb2AACA3WI+xZOamqqRI0dq4sSJqqys1IQJE/TMM89csG9hYaEk6dChQ5Ikr9erpqamqD7nX3u93ovu0+VyRe4cOr8AAICr12U/ByUcDqu1tfWCbXV1dZKk7OxsSZLP59P+/fvV3Nwc6bN9+3a53e7IaSIAAICYTvFUVFRo5syZysvL06lTp7Rhwwbt3LlTb7zxhg4fPqwNGzZo1qxZGjx4sPbt26elS5fq1ltvVUFBgSRp+vTpys/P17x587R69Wr5/X6tWLFCpaWlcrlcV+QAAQBA7xNTQGlubtb8+fPV2Ngoj8ejgoICvfHGG/r2t7+to0eP6s0339TTTz+tM2fOKDc3VyUlJVqxYkXk/UlJSdqyZYuWLFkin8+ngQMHasGCBVHPTQEAALjs56AkAs9BAQCg94nLc1AAAACuFAIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGCd5EQX0B3GGElSMBhMcCUAAKCrzn9un/8c70yvDCinTp2SJOXm5ia4EgAAEKtTp07J4/F02sdhuhJjLBMOh1VfX6/8/HwdPXpUbrc70SX1WsFgULm5uYxjD2Asew5j2TMYx57DWPYMY4xOnTqlnJwcOZ2dX2XSK2dQnE6nrr32WkmS2+3ml6UHMI49h7HsOYxlz2Acew5jefkuNXNyHhfJAgAA6xBQAACAdXptQHG5XFq5cqVcLleiS+nVGMeew1j2HMayZzCOPYexjL9eeZEsAAC4uvXaGRQAAHD1IqAAAADrEFAAAIB1CCgAAMA6vTKgrFmzRtddd5369eunwsJCvffee4kuyTq7du3S7bffrpycHDkcDr3yyitR7cYYPfLII8rOzlb//v1VVFSkTz75JKrPyZMnNXfuXLndbqWnp2vhwoU6ffp0HI8i8SorKzV58mQNGjRImZmZmjNnjurr66P6nDt3TqWlpRo8eLDS0tJUUlKipqamqD4NDQ2aPXu2BgwYoMzMTC1btkwdHR3xPJSEWrt2rQoKCiIPufL5fNq6dWuknTHsvieeeEIOh0MPPvhgZB3j2TWPPvqoHA5H1DJmzJhIO+OYYKaX2bhxo0lNTTX/+Z//aQ4ePGgWLVpk0tPTTVNTU6JLs8rrr79ufvKTn5iXX37ZSDKbN2+Oan/iiSeMx+Mxr7zyivnDH/5gvvOd75gRI0aYzz//PNJnxowZZsKECWb37t3md7/7nRk5cqS5995743wkiVVcXGxeeOEFc+DAAVNXV2dmzZpl8vLyzOnTpyN97r//fpObm2uqqqrMBx98YKZMmWL+9m//NtLe0dFhxo0bZ4qKiszevXvN66+/boYMGWIqKioScUgJ8d///d/mN7/5jfnjH/9o6uvrzY9//GOTkpJiDhw4YIxhDLvrvffeM9ddd50pKCgwP/jBDyLrGc+uWblypbnxxhtNY2NjZDl+/HiknXFMrF4XUG6++WZTWloaeR0KhUxOTo6prKxMYFV2+3JACYfDxuv1mieffDKyrqWlxbhcLvOLX/zCGGPMhx9+aCSZ999/P9Jn69atxuFwmL/85S9xq902zc3NRpKprq42xnwxbikpKWbTpk2RPh999JGRZGpqaowxX4RFp9Np/H5/pM/atWuN2+02ra2t8T0Ai1xzzTXmP/7jPxjDbjp16pQZNWqU2b59u/m7v/u7SEBhPLtu5cqVZsKECRdsYxwTr1ed4mlra1Ntba2Kiooi65xOp4qKilRTU5PAynqXI0eOyO/3R42jx+NRYWFhZBxramqUnp6uSZMmRfoUFRXJ6XRqz549ca/ZFoFAQJKUkZEhSaqtrVV7e3vUWI4ZM0Z5eXlRYzl+/HhlZWVF+hQXFysYDOrgwYNxrN4OoVBIGzdu1JkzZ+Tz+RjDbiotLdXs2bOjxk3idzJWn3zyiXJycnT99ddr7ty5amhokMQ42qBXfVngZ599plAoFPXLIElZWVn6+OOPE1RV7+P3+yXpguN4vs3v9yszMzOqPTk5WRkZGZE+fU04HNaDDz6oW265RePGjZP0xTilpqYqPT09qu+Xx/JCY32+ra/Yv3+/fD6fzp07p7S0NG3evFn5+fmqq6tjDGO0ceNG/f73v9f777//lTZ+J7uusLBQ69ev1+jRo9XY2KhVq1bpm9/8pg4cOMA4WqBXBRQgkUpLS3XgwAG9/fbbiS6lVxo9erTq6uoUCAT061//WgsWLFB1dXWiy+p1jh49qh/84Afavn27+vXrl+hyerWZM2dGfi4oKFBhYaGGDx+uX/3qV+rfv38CK4PUy+7iGTJkiJKSkr5yFXVTU5O8Xm+Cqup9zo9VZ+Po9XrV3Nwc1d7R0aGTJ0/2ybEuKyvTli1b9NZbb2nYsGGR9V6vV21tbWppaYnq/+WxvNBYn2/rK1JTUzVy5EhNnDhRlZWVmjBhgp555hnGMEa1tbVqbm7W3/zN3yg5OVnJycmqrq7Ws88+q+TkZGVlZTGe3ZSenq6vfe1rOnToEL+XFuhVASU1NVUTJ05UVVVVZF04HFZVVZV8Pl8CK+tdRowYIa/XGzWOwWBQe/bsiYyjz+dTS0uLamtrI3127NihcDiswsLCuNecKMYYlZWVafPmzdqxY4dGjBgR1T5x4kSlpKREjWV9fb0aGhqixnL//v1RgW/79u1yu93Kz8+Pz4FYKBwOq7W1lTGM0bRp07R//37V1dVFlkmTJmnu3LmRnxnP7jl9+rQOHz6s7Oxsfi9tkOirdGO1ceNG43K5zPr1682HH35oFi9ebNLT06OuosYXV/jv3bvX7N2710gy//Zv/2b27t1r/vznPxtjvrjNOD093bz66qtm37595o477rjgbcZf//rXzZ49e8zbb79tRo0a1eduM16yZInxeDxm586dUbcinj17NtLn/vvvN3l5eWbHjh3mgw8+MD6fz/h8vkj7+VsRp0+fburq6sy2bdvM0KFD+9StiA8//LCprq42R44cMfv27TMPP/ywcTgc5re//a0xhjG8XP/7Lh5jGM+ueuihh8zOnTvNkSNHzDvvvGOKiorMkCFDTHNzszGGcUy0XhdQjDHm5z//ucnLyzOpqanm5ptvNrt37050SdZ56623jKSvLAsWLDDGfHGr8U9/+lOTlZVlXC6XmTZtmqmvr4/axokTJ8y9995r0tLSjNvtNt///vfNqVOnEnA0iXOhMZRkXnjhhUifzz//3PzTP/2Tueaaa8yAAQPMd7/7XdPY2Bi1nT/96U9m5syZpn///mbIkCHmoYceMu3t7XE+msT5x3/8RzN8+HCTmppqhg4daqZNmxYJJ8YwhpfrywGF8eyau+++22RnZ5vU1FRz7bXXmrvvvtscOnQo0s44JpbDGGMSM3cDAABwYb3qGhQAANA3EFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYJ3/BymtJBHyfBqfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0.3646, 0.6354],\n",
       "         [0.4279, 0.5721]], grad_fn=<SoftmaxBackward0>),\n",
       " tensor([[0.2427],\n",
       "         [0.1591]], grad_fn=<AddmmBackward0>))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 2),\n",
    "    torch.nn.Softmax(dim=1),\n",
    ")\n",
    "\n",
    "model_td = sequential = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 1),\n",
    ")\n",
    "\n",
    "model(torch.randn(2, 4)), model_td(torch.randn(2, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "#得到一个动作\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1, 4)\n",
    "    #[1, 4] -> [1, 2]\n",
    "    prob = model(state)\n",
    "\n",
    "    #根据概率选择一个动作\n",
    "    action = random.choices(range(2), weights=prob[0].tolist(), k=1)[0]\n",
    "\n",
    "    return action\n",
    "\n",
    "\n",
    "get_action([1, 2, 3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1740/2726165283.py:31: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  states = torch.FloatTensor(states).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-8.6252e-03,  2.4674e-02,  4.0613e-02,  7.5287e-04],\n",
       "         [-8.1317e-03,  2.1919e-01,  4.0628e-02, -2.7884e-01],\n",
       "         [-3.7479e-03,  4.1371e-01,  3.5051e-02, -5.5844e-01],\n",
       "         [ 4.5263e-03,  2.1811e-01,  2.3882e-02, -2.5493e-01],\n",
       "         [ 8.8886e-03,  4.1289e-01,  1.8784e-02, -5.3998e-01],\n",
       "         [ 1.7146e-02,  2.1751e-01,  7.9843e-03, -2.4144e-01],\n",
       "         [ 2.1496e-02,  2.2271e-02,  3.1555e-03,  5.3752e-02],\n",
       "         [ 2.1942e-02, -1.7290e-01,  4.2305e-03,  3.4743e-01],\n",
       "         [ 1.8484e-02,  2.2166e-02,  1.1179e-02,  5.6083e-02],\n",
       "         [ 1.8927e-02,  2.1713e-01,  1.2301e-02, -2.3305e-01],\n",
       "         [ 2.3270e-02,  4.1207e-01,  7.6397e-03, -5.2183e-01],\n",
       "         [ 3.1511e-02,  2.1684e-01, -2.7969e-03, -2.2675e-01],\n",
       "         [ 3.5848e-02,  4.1200e-01, -7.3319e-03, -5.2031e-01],\n",
       "         [ 4.4088e-02,  2.1698e-01, -1.7738e-02, -2.2995e-01],\n",
       "         [ 4.8428e-02,  4.1236e-01, -2.2337e-02, -5.2817e-01],\n",
       "         [ 5.6675e-02,  6.0778e-01, -3.2901e-02, -8.2781e-01],\n",
       "         [ 6.8831e-02,  8.0334e-01, -4.9457e-02, -1.1307e+00],\n",
       "         [ 8.4897e-02,  9.9907e-01, -7.2070e-02, -1.4384e+00],\n",
       "         [ 1.0488e-01,  8.0491e-01, -1.0084e-01, -1.1691e+00],\n",
       "         [ 1.2098e-01,  1.0012e+00, -1.2422e-01, -1.4916e+00],\n",
       "         [ 1.4100e-01,  1.1976e+00, -1.5405e-01, -1.8204e+00],\n",
       "         [ 1.6495e-01,  1.3940e+00, -1.9046e-01, -2.1567e+00]]),\n",
       " tensor([[1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.]]),\n",
       " tensor([[1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1]]),\n",
       " tensor([[-0.0081,  0.2192,  0.0406, -0.2788],\n",
       "         [-0.0037,  0.4137,  0.0351, -0.5584],\n",
       "         [ 0.0045,  0.2181,  0.0239, -0.2549],\n",
       "         [ 0.0089,  0.4129,  0.0188, -0.5400],\n",
       "         [ 0.0171,  0.2175,  0.0080, -0.2414],\n",
       "         [ 0.0215,  0.0223,  0.0032,  0.0538],\n",
       "         [ 0.0219, -0.1729,  0.0042,  0.3474],\n",
       "         [ 0.0185,  0.0222,  0.0112,  0.0561],\n",
       "         [ 0.0189,  0.2171,  0.0123, -0.2331],\n",
       "         [ 0.0233,  0.4121,  0.0076, -0.5218],\n",
       "         [ 0.0315,  0.2168, -0.0028, -0.2267],\n",
       "         [ 0.0358,  0.4120, -0.0073, -0.5203],\n",
       "         [ 0.0441,  0.2170, -0.0177, -0.2299],\n",
       "         [ 0.0484,  0.4124, -0.0223, -0.5282],\n",
       "         [ 0.0567,  0.6078, -0.0329, -0.8278],\n",
       "         [ 0.0688,  0.8033, -0.0495, -1.1307],\n",
       "         [ 0.0849,  0.9991, -0.0721, -1.4384],\n",
       "         [ 0.1049,  0.8049, -0.1008, -1.1691],\n",
       "         [ 0.1210,  1.0012, -0.1242, -1.4916],\n",
       "         [ 0.1410,  1.1976, -0.1541, -1.8204],\n",
       "         [ 0.1650,  1.3940, -0.1905, -2.1567],\n",
       "         [ 0.1928,  1.5905, -0.2336, -2.5016]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1]]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_data():\n",
    "    states = []\n",
    "    rewards = []\n",
    "    actions = []\n",
    "    next_states = []\n",
    "    overs = []\n",
    "\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        next_state, reward, over, _ = env.step(action)\n",
    "\n",
    "        #记录数据样本\n",
    "        states.append(state)\n",
    "        rewards.append(reward)\n",
    "        actions.append(action)\n",
    "        next_states.append(next_state)\n",
    "        overs.append(over)\n",
    "\n",
    "        #更新游戏状态,开始下一个动作\n",
    "        state = next_state\n",
    "\n",
    "    #[b, 4]\n",
    "    states = torch.FloatTensor(states).reshape(-1, 4)\n",
    "    #[b, 1]\n",
    "    rewards = torch.FloatTensor(rewards).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    actions = torch.LongTensor(actions).reshape(-1, 1)\n",
    "    #[b, 4]\n",
    "    next_states = torch.FloatTensor(next_states).reshape(-1, 4)\n",
    "    #[b, 1]\n",
    "    overs = torch.LongTensor(overs).reshape(-1, 1)\n",
    "\n",
    "    return states, rewards, actions, next_states, overs\n",
    "\n",
    "\n",
    "get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 8251,
     "status": "ok",
     "timestamp": 1650011468229,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "BQXVYW2T_DcQ",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 29.7\n",
      "100 26.8\n",
      "200 56.2\n",
      "300 71.7\n",
      "400 165.7\n",
      "500 194.8\n",
      "600 192.3\n",
      "700 196.3\n",
      "800 200.0\n",
      "900 200.0\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n",
    "    optimizer_td = torch.optim.Adam(model_td.parameters(), lr=1e-2)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #玩N局游戏,每局游戏训练一次\n",
    "    for i in range(1000):\n",
    "        #玩一局游戏,得到数据\n",
    "        #states -> [b, 4]\n",
    "        #rewards -> [b, 1]\n",
    "        #actions -> [b, 1]\n",
    "        #next_states -> [b, 4]\n",
    "        #overs -> [b, 1]\n",
    "        states, rewards, actions, next_states, overs = get_data()\n",
    "\n",
    "        #计算values和targets\n",
    "        #[b, 4] -> [b ,1]\n",
    "        values = model_td(states)\n",
    "\n",
    "        #[b, 4] -> [b ,1]\n",
    "        targets = model_td(next_states) * 0.98\n",
    "        #[b ,1] * [b ,1] -> [b ,1]\n",
    "        targets *= (1 - overs)\n",
    "        #[b ,1] + [b ,1] -> [b ,1]\n",
    "        targets += rewards\n",
    "\n",
    "        #时序差分误差\n",
    "        #[b ,1] - [b ,1] -> [b ,1]\n",
    "        delta = (targets - values).detach()\n",
    "\n",
    "        #重新计算对应动作的概率\n",
    "        #[b, 4] -> [b ,2]\n",
    "        probs = model(states)\n",
    "        #[b ,2] -> [b ,1]\n",
    "        probs = probs.gather(dim=1, index=actions)\n",
    "\n",
    "        #根据策略梯度算法的导函数实现\n",
    "        #只是把公式中的reward_sum替换为了时序差分的误差\n",
    "        #[b ,1] * [b ,1] -> [b ,1] -> scala\n",
    "        loss = (-probs.log() * delta).mean()\n",
    "\n",
    "        #时序差分的loss就是简单的value和target求mse loss即可\n",
    "        loss_td = loss_fn(values, targets.detach())\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        optimizer_td.zero_grad()\n",
    "        loss_td.backward()\n",
    "        optimizer_td.step()\n",
    "\n",
    "        if i % 100 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(i, test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqFklEQVR4nO3df3RU9Z3/8ddMkhkSwkwaIJmkJIiCYIBgCxpmbS0uKQHRlRrPUUsFXY4c2cRTjbWYrlWxe4zF7fqjq/A9u1txz5HS2q/oSgWLQcJaI2Ik5ZemwtIGhUlQmkwSyCSZ+Xz/cJlvRwPJJJPMnfh8nHOPM/fzmTvv+zk5zovP/WUzxhgBAABYiD3eBQAAAHweAQUAAFgOAQUAAFgOAQUAAFgOAQUAAFgOAQUAAFgOAQUAAFgOAQUAAFgOAQUAAFgOAQUAAFhOXAPK008/rQsuuECjRo1SUVGR3nnnnXiWAwAALCJuAeVXv/qVKioq9OCDD+q9997TrFmzVFJSoubm5niVBAAALMIWr4cFFhUV6bLLLtO//uu/SpJCoZDy8vJ055136r777otHSQAAwCKS4/GlXV1dqqurU2VlZXid3W5XcXGxamtrv9A/EAgoEAiE34dCIZ06dUpjx46VzWYblpoBAMDgGGPU1tam3Nxc2e3nP4gTl4DyySefKBgMKjs7O2J9dna2Pvjggy/0r6qq0po1a4arPAAAMISOHTumCRMmnLdPXAJKtCorK1VRURF+39raqvz8fB07dkwulyuOlQEAgP7y+/3Ky8vTmDFj+uwbl4Aybtw4JSUlqampKWJ9U1OTPB7PF/o7nU45nc4vrHe5XAQUAAASTH9Oz4jLVTwOh0OzZ89WdXV1eF0oFFJ1dbW8Xm88SgIAABYSt0M8FRUVWr58uebMmaPLL79cTzzxhDo6OnTbbbfFqyQAAGARcQsoN954o06ePKkHHnhAPp9Pl156qbZt2/aFE2cBAMCXT9zugzIYfr9fbrdbra2tnIMCAECCiOb3m2fxAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAy4l5QHnooYdks9kilmnTpoXbOzs7VVZWprFjxyo9PV2lpaVqamqKdRkAACCBDckMyvTp03XixInw8uabb4bb7r77br3yyit64YUXVFNTo+PHj+v6668fijIAAECCSh6SjSYny+PxfGF9a2ur/uM//kMbN27U3/7t30qSnn32WV1yySV6++23NXfu3KEoBwAAJJghmUH58MMPlZubqwsvvFBLly5VY2OjJKmurk7d3d0qLi4O9502bZry8/NVW1t7zu0FAgH5/f6IBQAAjFwxDyhFRUXasGGDtm3bpnXr1uno0aP65je/qba2Nvl8PjkcDmVkZER8Jjs7Wz6f75zbrKqqktvtDi95eXmxLhsAAFhIzA/xLFq0KPy6sLBQRUVFmjhxon79618rNTV1QNusrKxURUVF+L3f7yekAAAwgg35ZcYZGRm6+OKLdfjwYXk8HnV1damlpSWiT1NTU6/nrJzldDrlcrkiFgAAMHINeUBpb2/XkSNHlJOTo9mzZyslJUXV1dXh9oaGBjU2Nsrr9Q51KQAAIEHE/BDPD37wA1177bWaOHGijh8/rgcffFBJSUm6+eab5Xa7tWLFClVUVCgzM1Mul0t33nmnvF4vV/AAAICwmAeUjz76SDfffLM+/fRTjR8/Xt/4xjf09ttva/z48ZKkxx9/XHa7XaWlpQoEAiopKdEzzzwT6zIAAEACsxljTLyLiJbf75fb7VZrayvnowAAkCCi+f3mWTwAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByog4ou3bt0rXXXqvc3FzZbDa99NJLEe3GGD3wwAPKyclRamqqiouL9eGHH0b0OXXqlJYuXSqXy6WMjAytWLFC7e3tg9oRAAAwckQdUDo6OjRr1iw9/fTTvbavXbtWTz31lNavX6/du3dr9OjRKikpUWdnZ7jP0qVLdfDgQW3fvl1btmzRrl27tHLlyoHvBQAAGFFsxhgz4A/bbNq8ebOWLFki6bPZk9zcXN1zzz36wQ9+IElqbW1Vdna2NmzYoJtuuknvv/++CgoKtGfPHs2ZM0eStG3bNl199dX66KOPlJub2+f3+v1+ud1utba2yuVyDbR8AAAwjKL5/Y7pOShHjx6Vz+dTcXFxeJ3b7VZRUZFqa2slSbW1tcrIyAiHE0kqLi6W3W7X7t27e91uIBCQ3++PWAAAwMgV04Di8/kkSdnZ2RHrs7Ozw20+n09ZWVkR7cnJycrMzAz3+byqqiq53e7wkpeXF8uyAQCAxSTEVTyVlZVqbW0NL8eOHYt3SQAAYAjFNKB4PB5JUlNTU8T6pqamcJvH41Fzc3NEe09Pj06dOhXu83lOp1MulytiAQAAI1dMA8qkSZPk8XhUXV0dXuf3+7V79255vV5JktfrVUtLi+rq6sJ9duzYoVAopKKioliWAwAAElRytB9ob2/X4cOHw++PHj2q+vp6ZWZmKj8/X3fddZf+6Z/+SVOmTNGkSZP04x//WLm5ueErfS655BItXLhQt99+u9avX6/u7m6Vl5frpptu6tcVPAAAYOSLOqC8++67uuqqq8LvKyoqJEnLly/Xhg0b9MMf/lAdHR1auXKlWlpa9I1vfEPbtm3TqFGjwp95/vnnVV5ervnz58tut6u0tFRPPfVUDHYHAACMBIO6D0q8cB8UAAAST9zugwIAABALBBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5BBQAAGA5UQeUXbt26dprr1Vubq5sNpteeumliPZbb71VNpstYlm4cGFEn1OnTmnp0qVyuVzKyMjQihUr1N7ePqgdAQAAI0fUAaWjo0OzZs3S008/fc4+Cxcu1IkTJ8LLL3/5y4j2pUuX6uDBg9q+fbu2bNmiXbt2aeXKldFXDwAARqTkaD+waNEiLVq06Lx9nE6nPB5Pr23vv/++tm3bpj179mjOnDmSpJ///Oe6+uqr9c///M/Kzc2NtiQAADDCDMk5KDt37lRWVpamTp2qVatW6dNPPw231dbWKiMjIxxOJKm4uFh2u127d+/udXuBQEB+vz9iAQAAI1fMA8rChQv1n//5n6qurtZPf/pT1dTUaNGiRQoGg5Ikn8+nrKysiM8kJycrMzNTPp+v121WVVXJ7XaHl7y8vFiXDQAALCTqQzx9uemmm8KvZ86cqcLCQl100UXauXOn5s+fP6BtVlZWqqKiIvze7/cTUgAAGMGG/DLjCy+8UOPGjdPhw4clSR6PR83NzRF9enp6dOrUqXOet+J0OuVyuSIWAAAwcg15QPnoo4/06aefKicnR5Lk9XrV0tKiurq6cJ8dO3YoFAqpqKhoqMsBAAAJIOpDPO3t7eHZEEk6evSo6uvrlZmZqczMTK1Zs0alpaXyeDw6cuSIfvjDH2ry5MkqKSmRJF1yySVauHChbr/9dq1fv17d3d0qLy/XTTfdxBU8AABAkmQzxphoPrBz505dddVVX1i/fPlyrVu3TkuWLNHevXvV0tKi3NxcLViwQD/5yU+UnZ0d7nvq1CmVl5frlVdekd1uV2lpqZ566imlp6f3qwa/3y+3263W1lYO9wAAkCCi+f2OOqBYAQEFAIDEE83vN8/iAQAAlkNAAQAAlkNAAQAAlkNAAQAAlkNAAQAAlkNAAQAAlkNAAQAAlkNAAQAAlkNAAQAAlkNAAQAAlhP1wwIB4POMMerpbFfPmTZ1d7ap50y7us/41XPGr+7OdvWc8WvC5d/RqAxPvEsFkCAIKAAG7f2Xfqqu9k9lTEgmFJL+97+fvQ9KJqTx06+S050tm80W73IBJAACCoBB6z7dou7TreftE+w6I8lIIqAA6BvnoAAYFsHA6c/yCQD0AwEFwLD4/zMoANA3AgqAYdETOC1jCCgA+oeAAmBYBAMdYgYFQH8RUAAMWvKoMX32aT74hkLBnmGoBsBIQEABMGjjpl0R7xIAjDAEFACDluxME5cPA4glAgqAQUtypMW7BAAjDAEFwKAlO0fHuwQAIwwBBcCgJTmZQQEQWwQUAIOW7Ezr3ykoJjTktQAYGQgoAAYtKWVUv/oFuwNDXAmAkYKAAmDw+vmE4mDg9BAXAmCkIKAAGDY9gY54lwAgQRBQAAybHmZQAPQTAQXAsOEQD4D+IqAAGDbMoADor6gCSlVVlS677DKNGTNGWVlZWrJkiRoaGiL6dHZ2qqysTGPHjlV6erpKS0vV1NQU0aexsVGLFy9WWlqasrKydO+996qnh4eIAYmt7xNlTx56YxjqADASRBVQampqVFZWprffflvbt29Xd3e3FixYoI6O/3/i2913361XXnlFL7zwgmpqanT8+HFdf/314fZgMKjFixerq6tLb731lp577jlt2LBBDzzwQOz2CsCwstns8sxa0Ge/nk5mUAD0j80YYwb64ZMnTyorK0s1NTW68sor1draqvHjx2vjxo264YYbJEkffPCBLrnkEtXW1mru3LnaunWrrrnmGh0/flzZ2dmSpPXr12v16tU6efKkHA5Hn9/r9/vldrvV2toql8s10PIBxIgxIfn+8Lo+2v2b8/ZLcqTp67c9MTxFAbCcaH6/B3UOSmtrqyQpMzNTklRXV6fu7m4VFxeH+0ybNk35+fmqra2VJNXW1mrmzJnhcCJJJSUl8vv9OnjwYK/fEwgE5Pf7IxYAVmJTsjM13kUAGEEGHFBCoZDuuusuXXHFFZoxY4YkyefzyeFwKCMjI6Jvdna2fD5fuM9fh5Oz7WfbelNVVSW32x1e8vLyBlo2gCHC83gAxNKAA0pZWZkOHDigTZs2xbKeXlVWVqq1tTW8HDt2bMi/E0B0kniiMYAYSh7Ih8rLy7Vlyxbt2rVLEyZMCK/3eDzq6upSS0tLxCxKU1OTPB5PuM8777wTsb2zV/mc7fN5TqdTTqdzIKUCGCbJjv4d4jHGyNbPW+MD+PKKagbFGKPy8nJt3rxZO3bs0KRJkyLaZ8+erZSUFFVXV4fXNTQ0qLGxUV6vV5Lk9Xq1f/9+NTc3h/ts375dLpdLBQUFg9kXAHFis9mUlNKff0QYHhgIoF+imkEpKyvTxo0b9fLLL2vMmDHhc0bcbrdSU1Pldru1YsUKVVRUKDMzUy6XS3feeae8Xq/mzp0rSVqwYIEKCgp0yy23aO3atfL5fLr//vtVVlbGLAkw0hmjYNdpJTv69/RjAF9eUQWUdevWSZLmzZsXsf7ZZ5/VrbfeKkl6/PHHZbfbVVpaqkAgoJKSEj3zzDPhvklJSdqyZYtWrVolr9er0aNHa/ny5Xr44YcHtycALM9ICnadiXcZABLAoO6DEi/cBwWwns4Wn/b/6vw3XLQnO3Xx4u9rjGfyMFUFwEqG7T4oABAdwwwKgH4hoAAYNsYQUAD0DwEFQEzYU0YpbdzE8/YxoaD8H38wTBUBSGQEFAAxYU92KDXzq+fvZELqbOn9jtEA8NcIKABiwmazKYnLhwHECAEFQGzY7EpKIaAAiA0CCoCYsNnssvfrbrIA0DcCCoCYsNmZQQEQOwQUALFhs/fvHBRjlID3hwQwzAgoAGLGntT30zOMCSnU0zUM1QBIZAQUADFhs9n61c+EggQUAH0ioAAYViYUVLA7EO8yAFgcAQXAsDKhkEI9BBQA50dAATCsTCioEDMoAPpAQAEwrELBbvV0tse7DAAWR0ABEDNJjlQlO0eft093x1/k/+jQMFUEIFERUADEjNOVpdRxefEuA8AIQEABEDO2pGTZkxzxLgPACEBAARAzdnuy7Mkp8S4DwAhAQAEQM7akZNmTmUEBMHgEFAAx89khHmZQAAweAQVAzPT3EI8RDwwEcH4EFAAxY7PbZbP1/b8VE+yRCfUMQ0UAEhUBBcCwC/V0K9RDQAFwbgQUAMMuFOxmBgXAeRFQAAy7UE+3TLA73mUAsDACCoBhZ4JdCgWZQQFwbgQUAMMuFOwhoAA4LwIKgJhyTSiQI33sefu0N/2Pznz60TBVBCAREVAAxFSSI1X2pOTzdzIhGRManoIAJCQCCoCYsic7JHtSvMsAkOCiCihVVVW67LLLNGbMGGVlZWnJkiVqaGiI6DNv3jzZbLaI5Y477ojo09jYqMWLFystLU1ZWVm699571cM9EYARwZ7ilJ2AAmCQ+piHjVRTU6OysjJddtll6unp0Y9+9CMtWLBAhw4d0ujRo8P9br/9dj388MPh92lpaeHXwWBQixcvlsfj0VtvvaUTJ05o2bJlSklJ0SOPPBKDXQIQT/Zkh2wEFACDFFVA2bZtW8T7DRs2KCsrS3V1dbryyivD69PS0uTxeHrdxu9+9zsdOnRIr7/+urKzs3XppZfqJz/5iVavXq2HHnpIDgdPQgUSWVKyk4ACYNAGdQ5Ka2urJCkzMzNi/fPPP69x48ZpxowZqqys1OnTp8NttbW1mjlzprKzs8PrSkpK5Pf7dfDgwV6/JxAIyO/3RywArMmenNK/gGJ4YCCAc4tqBuWvhUIh3XXXXbriiis0Y8aM8Prvfve7mjhxonJzc7Vv3z6tXr1aDQ0NevHFFyVJPp8vIpxICr/3+Xy9fldVVZXWrFkz0FIBDCObPUmy2frsF+wJSCYk2ZhtAfBFAw4oZWVlOnDggN58882I9StXrgy/njlzpnJycjR//nwdOXJEF1100YC+q7KyUhUVFeH3fr9feXl5AyscgCWEujtlTEg2EVAAfNGADvGUl5dry5YteuONNzRhwoTz9i0qKpIkHT58WJLk8XjU1NQU0efs+3Odt+J0OuVyuSIWAIkt2N0pE+JeKAB6F1VAMcaovLxcmzdv1o4dOzRp0qQ+P1NfXy9JysnJkSR5vV7t379fzc3N4T7bt2+Xy+VSQUFBNOUASGDBrv89xAMAvYjqEE9ZWZk2btyol19+WWPGjAmfM+J2u5WamqojR45o48aNuvrqqzV27Fjt27dPd999t6688koVFhZKkhYsWKCCggLdcsstWrt2rXw+n+6//36VlZXJ6XTGfg8BWFKIGRQA5xHVDMq6devU2tqqefPmKScnJ7z86le/kiQ5HA69/vrrWrBggaZNm6Z77rlHpaWleuWVV8LbSEpK0pYtW5SUlCSv16vvfe97WrZsWcR9UwCMfB0n/6Rg95l4lwHAoqKaQenrksC8vDzV1NT0uZ2JEyfq1VdfjearASQQd/5MtTf9j0yw+5x9zpz6WMHuwDBWBSCR8CweADGXmpHDzdoADAoBBUDMJTlTZevHvVAA4FwIKABiLiklVRIBBcDAEVAAxFySI7Vfd5MFgHMhoACIuSTHKA7xABgUAgqAmEty9O8QjwmFeGAggF4RUADEXH9nT4Jdp/vuBOBLiYACIG6CAW7UBqB3BBQAcdPTdVoSh3gAfBEBBUDcBAOnyScAekVAARA3wS4O8QDoHQEFwJBwjM7os8+ZFp+YQgHQGwIKgCEx9mJvn33+cmSPTCg0DNUASDQEFABDInnU6HiXACCBEVAADIkkR1q8SwCQwAgoAIZEspMZFAADR0ABMCSSnMygABg4AgqAIcEMCoDBIKAAGBJJKc5+9TOhniGuBEAiSo53AQCsKxgMDvhpw6F+Xj4cON0uhz1lQN9xlt1ul93Ov7eAkYSAAuCcvF6v9u7dO6DPjnIkq/pn3zvvk42NMfr6rOk6cvwvAy1RkvRv//ZvuvXWWwe1DQDWQkABcE7BYFA9PQM7BNPTzwmNVEfSgL/jrIHO8gCwLgIKgGFxqtujlu4sBU2KnEntGp/ykUbZ2+VKc8S7NAAWREABMOT+dGa6/nxmujpD6QopScm2Ln2c1KJZY6o1ZnT/TqYF8OXCWWUAhkwwJH3cOVl/7Lhcp0MZCilZkk09xqmWnmzVtixR+ugx8S4TgAURUAAMie6eoP7Pax9rX/tVCqr3q3QCoVRN+PpPhrkyAImAgAJgyLSd6ZJ07qt4PrvC59ztAL68CCgAhoQxUvvprniXASBBEVAADAkjozYCCoABIqAAGBpGsnWfUMHoN2VTsNcuSerWFRkvDnNhABJBVAFl3bp1KiwslMvlksvlktfr1datW8PtnZ2dKisr09ixY5Wenq7S0lI1NTVFbKOxsVGLFy9WWlqasrKydO+99w76Jk0ArMdIaj/dqfxRhzQlrU5Oe/v/BhWjJHVrdNJfdMVX/q+c9tPxLhWABUV1H5QJEybo0Ucf1ZQpU2SM0XPPPafrrrtOe/fu1fTp03X33Xfrt7/9rV544QW53W6Vl5fr+uuv1+9//3tJn92VcvHixfJ4PHrrrbd04sQJLVu2TCkpKXrkkUeGZAcBxE/TXzr08u8/kPSBTnZN0KnuXPUYh1LtbfI4/0enkto4TwVAr2xmkPeIzszM1GOPPaYbbrhB48eP18aNG3XDDTdIkj744ANdcsklqq2t1dy5c7V161Zdc801On78uLKzsyVJ69ev1+rVq3Xy5Ek5HP27o6Tf75fb7datt97a788AiN6LL76oTz75JN5l9Olb3/qWpk6dGu8yAPShq6tLGzZsUGtrq1wu13n7DvhOssFgUC+88II6Ojrk9XpVV1en7u5uFRcXh/tMmzZN+fn54YBSW1urmTNnhsOJJJWUlGjVqlU6ePCgvva1r/X6XYFAQIFAIPze7/dLkm655Ralp6cPdBcA9KGmpiYhAsqVV16pa665Jt5lAOhDe3u7NmzY0K++UQeU/fv3y+v1qrOzU+np6dq8ebMKCgpUX18vh8OhjIyMiP7Z2dny+XySJJ/PFxFOzrafbTuXqqoqrVmz5gvr58yZ02cCAzBwo0ePjncJ/TJp0iRdfvnl8S4DQB/OTjD0R9RX8UydOlX19fXavXu3Vq1apeXLl+vQoUPRbiYqlZWVam1tDS/Hjh0b0u8DAADxFfUMisPh0OTJkyVJs2fP1p49e/Tkk0/qxhtvVFdXl1paWiJmUZqamuTxeCRJHo9H77zzTsT2zl7lc7ZPb5xOp5xOHigGAMCXxaDvgxIKhRQIBDR79mylpKSouro63NbQ0KDGxkZ5vV5Jktfr1f79+9Xc3Bzus337drlcLhUUFAy2FAAAMEJENYNSWVmpRYsWKT8/X21tbdq4caN27typ1157TW63WytWrFBFRYUyMzPlcrl05513yuv1au7cuZKkBQsWqKCgQLfccovWrl0rn8+n+++/X2VlZcyQAACAsKgCSnNzs5YtW6YTJ07I7XarsLBQr732mr797W9Lkh5//HHZ7XaVlpYqEAiopKREzzzzTPjzSUlJ2rJli1atWiWv16vRo0dr+fLlevjhh2O7VwAAIKEN+j4o8XD2Pij9uY4awMDNnj1b7733XrzL6NMvfvEL3XbbbfEuA0Afovn95lk8AADAcggoAADAcggoAADAcggoAADAcgb8LB4AI9+8efOUn58f7zL6lAg1AogOAQXAOf3sZz+LdwkAvqQ4xAMAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACwnqoCybt06FRYWyuVyyeVyyev1auvWreH2efPmyWazRSx33HFHxDYaGxu1ePFipaWlKSsrS/fee696enpiszcAAGBESI6m84QJE/Too49qypQpMsboueee03XXXae9e/dq+vTpkqTbb79dDz/8cPgzaWlp4dfBYFCLFy+Wx+PRW2+9pRMnTmjZsmVKSUnRI488EqNdAgAAic5mjDGD2UBmZqYee+wxrVixQvPmzdOll16qJ554ote+W7du1TXXXKPjx48rOztbkrR+/XqtXr1aJ0+elMPh6Nd3+v1+ud1utba2yuVyDaZ8AAAwTKL5/R7wOSjBYFCbNm1SR0eHvF5veP3zzz+vcePGacaMGaqsrNTp06fDbbW1tZo5c2Y4nEhSSUmJ/H6/Dh48eM7vCgQC8vv9EQsAABi5ojrEI0n79++X1+tVZ2en0tPTtXnzZhUUFEiSvvvd72rixInKzc3Vvn37tHr1ajU0NOjFF1+UJPl8vohwIin83ufznfM7q6qqtGbNmmhLBQAACSrqgDJ16lTV19ertbVVv/nNb7R8+XLV1NSooKBAK1euDPebOXOmcnJyNH/+fB05ckQXXXTRgIusrKxURUVF+L3f71deXt6AtwcAAKwt6kM8DodDkydP1uzZs1VVVaVZs2bpySef7LVvUVGRJOnw4cOSJI/Ho6ampog+Z997PJ5zfqfT6QxfOXR2AQAAI9eg74MSCoUUCAR6bauvr5ck5eTkSJK8Xq/279+v5ubmcJ/t27fL5XKFDxMBAABEdYinsrJSixYtUn5+vtra2rRx40bt3LlTr732mo4cOaKNGzfq6quv1tixY7Vv3z7dfffduvLKK1VYWChJWrBggQoKCnTLLbdo7dq18vl8uv/++1VWVian0zkkOwgAABJPVAGlublZy5Yt04kTJ+R2u1VYWKjXXntN3/72t3Xs2DG9/vrreuKJJ9TR0aG8vDyVlpbq/vvvD38+KSlJW7Zs0apVq+T1ejV69GgtX7484r4pAAAAg74PSjxwHxQAABLPsNwHBQAAYKgQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUkx7uAgTDGSJL8fn+cKwEAAP119nf77O/4+SRkQGlra5Mk5eXlxbkSAAAQrba2Nrnd7vP2sZn+xBiLCYVCamhoUEFBgY4dOyaXyxXvkhKW3+9XXl4e4xgDjGXsMJaxwTjGDmMZG8YYtbW1KTc3V3b7+c8yScgZFLvdrq9+9auSJJfLxR9LDDCOscNYxg5jGRuMY+wwloPX18zJWZwkCwAALIeAAgAALCdhA4rT6dSDDz4op9MZ71ISGuMYO4xl7DCWscE4xg5jOfwS8iRZAAAwsiXsDAoAABi5CCgAAMByCCgAAMByCCgAAMByEjKgPP3007rgggs0atQoFRUV6Z133ol3SZaza9cuXXvttcrNzZXNZtNLL70U0W6M0QMPPKCcnBylpqaquLhYH374YUSfU6dOaenSpXK5XMrIyNCKFSvU3t4+jHsRf1VVVbrssss0ZswYZWVlacmSJWpoaIjo09nZqbKyMo0dO1bp6ekqLS1VU1NTRJ/GxkYtXrxYaWlpysrK0r333quenp7h3JW4WrdunQoLC8M3ufJ6vdq6dWu4nTEcuEcffVQ2m0133XVXeB3j2T8PPfSQbDZbxDJt2rRwO+MYZybBbNq0yTgcDvOLX/zCHDx40Nx+++0mIyPDNDU1xbs0S3n11VfNP/7jP5oXX3zRSDKbN2+OaH/00UeN2+02L730kvnDH/5g/u7v/s5MmjTJnDlzJtxn4cKFZtasWebtt982//3f/20mT55sbr755mHek/gqKSkxzz77rDlw4ICpr683V199tcnPzzft7e3hPnfccYfJy8sz1dXV5t133zVz5841f/M3fxNu7+npMTNmzDDFxcVm79695tVXXzXjxo0zlZWV8diluPiv//ov89vf/tb88Y9/NA0NDeZHP/qRSUlJMQcOHDDGMIYD9c4775gLLrjAFBYWmu9///vh9Yxn/zz44INm+vTp5sSJE+Hl5MmT4XbGMb4SLqBcfvnlpqysLPw+GAya3NxcU1VVFceqrO3zASUUChmPx2Mee+yx8LqWlhbjdDrNL3/5S2OMMYcOHTKSzJ49e8J9tm7damw2m/n444+HrXaraW5uNpJMTU2NMeazcUtJSTEvvPBCuM/7779vJJna2lpjzGdh0W63G5/PF+6zbt0643K5TCAQGN4dsJCvfOUr5t///d8ZwwFqa2szU6ZMMdu3bzff+ta3wgGF8ey/Bx980MyaNavXNsYx/hLqEE9XV5fq6upUXFwcXme321VcXKza2to4VpZYjh49Kp/PFzGObrdbRUVF4XGsra1VRkaG5syZE+5TXFwsu92u3bt3D3vNVtHa2ipJyszMlCTV1dWpu7s7YiynTZum/Pz8iLGcOXOmsrOzw31KSkrk9/t18ODBYazeGoLBoDZt2qSOjg55vV7GcIDKysq0ePHiiHGT+JuM1ocffqjc3FxdeOGFWrp0qRobGyUxjlaQUA8L/OSTTxQMBiP+GCQpOztbH3zwQZyqSjw+n0+Seh3Hs20+n09ZWVkR7cnJycrMzAz3+bIJhUK66667dMUVV2jGjBmSPhsnh8OhjIyMiL6fH8vexvps25fF/v375fV61dnZqfT0dG3evFkFBQWqr69nDKO0adMmvffee9qzZ88X2vib7L+ioiJt2LBBU6dO1YkTJ7RmzRp985vf1IEDBxhHC0iogALEU1lZmQ4cOKA333wz3qUkpKlTp6q+vl6tra36zW9+o+XLl6umpibeZSWcY8eO6fvf/762b9+uUaNGxbuchLZo0aLw68LCQhUVFWnixIn69a9/rdTU1DhWBinBruIZN26ckpKSvnAWdVNTkzweT5yqSjxnx+p84+jxeNTc3BzR3tPTo1OnTn0px7q8vFxbtmzRG2+8oQkTJoTXezwedXV1qaWlJaL/58eyt7E+2/Zl4XA4NHnyZM2ePVtVVVWaNWuWnnzyScYwSnV1dWpubtbXv/51JScnKzk5WTU1NXrqqaeUnJys7OxsxnOAMjIydPHFF+vw4cP8XVpAQgUUh8Oh2bNnq7q6OrwuFAqpurpaXq83jpUllkmTJsnj8USMo9/v1+7du8Pj6PV61dLSorq6unCfHTt2KBQKqaioaNhrjhdjjMrLy7V582bt2LFDkyZNimifPXu2UlJSIsayoaFBjY2NEWO5f//+iMC3fft2uVwuFRQUDM+OWFAoFFIgEGAMozR//nzt379f9fX14WXOnDlaunRp+DXjOTDt7e06cuSIcnJy+Lu0gnifpRutTZs2GafTaTZs2GAOHTpkVq5caTIyMiLOosZnZ/jv3bvX7N2710gy//Iv/2L27t1r/vznPxtjPrvMOCMjw7z88stm37595rrrruv1MuOvfe1rZvfu3ebNN980U6ZM+dJdZrxq1SrjdrvNzp07Iy5FPH36dLjPHXfcYfLz882OHTvMu+++a7xer/F6veH2s5ciLliwwNTX15tt27aZ8ePHf6kuRbzvvvtMTU2NOXr0qNm3b5+57777jM1mM7/73e+MMYzhYP31VTzGMJ79dc8995idO3eao0ePmt///vemuLjYjBs3zjQ3NxtjGMd4S7iAYowxP//5z01+fr5xOBzm8ssvN2+//Xa8S7KcN954w0j6wrJ8+XJjzGeXGv/4xz822dnZxul0mvnz55uGhoaIbXz66afm5ptvNunp6cblcpnbbrvNtLW1xWFv4qe3MZRknn322XCfM2fOmH/4h38wX/nKV0xaWpr5zne+Y06cOBGxnT/96U9m0aJFJjU11YwbN87cc889pru7e5j3Jn7+/u//3kycONE4HA4zfvx4M3/+/HA4MYYxHKzPBxTGs39uvPFGk5OTYxwOh/nqV79qbrzxRnP48OFwO+MYXzZjjInP3A0AAEDvEuocFAAA8OVAQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJbz/wByQDW+RrUamwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第9章-策略梯度算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
